#!/usr/bin/env python

# (c) 2007-2011 Helsinki University of Technology and University of Helsinki
# Licensed under the MIT license; see LICENSE.txt for more information.

from harava.worker import Worker
from harava.rdf import *

import sys
import re
import urlparse
import lxml.etree

# control characters that are illegal in XML
CTRLCHARS = re.compile(u'[\x00-\x08\x0b\x0c\x0e-\x1f\ud800-\udfff\ufffe\uffff]')

class DCHTMLImporter (Worker):
  def __init__(self, session):
    Worker.__init__(self, session.getFatalEvent())
    self._session = session

  def process(self, doc):
    tree = doc.getParseTree()
    
    if tree is None:
      doc.processingFinished()
      return False
  
    lookup = self._session.getLookupService()
    # <link> elements
    for link in tree.getiterator('link'):
      fld = link.attrib.get('rel', '').lower()
      if not fld: continue
      href = URIRef(link.attrib.get('href', ''))
      if not href: continue
      
      if fld in ('dc.subject', 'dcterms.subject'):
        doc.setMetadata(DC.subject, href)
      elif fld in ('ts.genre', 'dcterms.genre'):
        doc.setMetadata(TS.genre, href)
      elif fld == 'dc.rights':
        doc.setMetadata(DC.rights, Literal(href))
      elif fld == 'dcterms.audience':
        doc.setMetadata(DCTERMS.audience, href)
    
    # <meta> elements
    for meta in tree.getiterator('meta'):
      if not meta.attrib.has_key('name'): continue	# skip http-equiv meta elements
      
      fld = meta.attrib.get('name', '').lower()
      if not fld: continue
      content = meta.attrib.get('content', '')
      if not content: continue				# skip empty meta elements
      
      # check for illegal control characters
      ctrl = CTRLCHARS.findall(content)
      if ctrl:
        doc.log('WARN', 'import', 'Illegal control characters found in <meta> element', ''.join(ctrl))
        content = CTRLCHARS.sub('?', content)
      
      lang = meta.attrib.get('lang', None) or meta.attrib.get('{http://www.w3.org/XML/1998/namespace}lang', None)
      scheme = meta.attrib.get('scheme', '')
      if scheme: scheme = scheme.lower()

      # dc:identifier
      if fld == 'dc.identifier':
        if self._session.getSource().getForceURLasURI():
          doc.log('NOTE', 'import', 'Forced URL as identifier instead of given value', content)
        else:
          if not scheme:
            doc.log('NOTE', 'import', 'No identifier scheme given, using DCTERMS.URI')
          elif scheme == 'url':
            doc.log('NOTE', 'import', 'Deprecated identifier scheme, using DCTERMS.URI', scheme)
          elif scheme != 'dcterms.uri':
            doc.log('WARN', 'import', 'Unknown identifier scheme', scheme)
          doc.setMetadata(DC.identifier, URIRef(content))

      # ts:url
      elif fld == 'ts.url':
        if content.startswith('http://') or content.startswith('https://'):
          doc.setMetadata(TS.url, URIRef(content))
        else:
          doc.log('NOTE', 'import', 'Invalid URL address', content)

      # dc:title
      elif fld == 'dc.title':
        if content.startswith('KTL | '): content = content.replace('KTL | ', '')	# KTL hack
        doc.setMetadata(DC['title'], Literal(content, lang))

      # dc:description
      elif fld in ('dcterms.abstract', 'dc.description', 'dc.description.abstract'):
        if fld != 'dc.description' and fld != 'dcterms.abstract':
          doc.log('NOTE', 'import', 'Deprecated field name', meta.attrib.get('name'))
        doc.setMetadata(DC.description, Literal(content, lang))

      # dc:language
      elif fld == 'dc.language':
        if not scheme:
          doc.log('NOTE', 'import', 'No language scheme given, using DCTERMS.RFC4646')
          scheme = 'dcterms.rfc4646'
        elif scheme not in ('dcterms.rfc4646', 'dcterms.rfc3066'):
          doc.log('WARN', 'import', 'Unknown language scheme', scheme)
        doc.setMetadata(DC.language, Literal(content, datatype=DCTERMS.RFC4646))

      # dcterms:issued
      elif fld in ('dcterms.issued', 'dc.date.created', 'dc.date.issued'):
        if fld != 'dcterms.issued':
          doc.log('NOTE', 'import', 'Deprecated field name', meta.attrib.get('name'))
        if not scheme:
          doc.log('NOTE', 'import', 'No date scheme given, using DCTERMS.W3CDTF')
          scheme = 'dcterms.w3cdtf'
        elif scheme == 'iso8601':
          doc.log('NOTE', 'import', 'Deprecated date scheme', scheme)
        elif scheme != 'dcterms.w3cdtf':
          doc.log('WARN', 'import', 'Unknown date scheme', scheme)
        doc.setMetadata(DCTERMS.issued, Literal(content, datatype=DCTERMS.W3CDTF))

      # dcterms:dateAccepted
      elif fld in ('dcterms.dateaccepted', 'dc.date.accepted', 'dc.date.dateaccepted'):
        if fld != 'dcterms.dateaccepted':
          doc.log('NOTE', 'import', 'Deprecated field name', meta.attrib.get('name'))
        if not scheme:
          doc.log('NOTE', 'import', 'No date scheme given, using DCTERMS.W3CDTF')
          scheme = 'dcterms.w3cdtf'
        elif scheme == 'iso8601':
          doc.log('NOTE', 'import', 'Deprecated date scheme', scheme)
        elif scheme != 'dcterms.w3cdtf':
          doc.log('WARN', 'import', 'Unknown date scheme', scheme)
        doc.setMetadata(DCTERMS.dateAccepted, Literal(content, datatype=DCTERMS.W3CDTF))

      # dcterms:modified
      elif fld in ('dcterms.modified', 'dc.date.modified'):
        if fld != 'dcterms.modified':
          doc.log('NOTE', 'import', 'Deprecated field name', meta.attrib.get('name'))
        if not scheme:
          doc.log('NOTE', 'import', 'No date scheme given, using DCTERMS.W3CDTF')
          scheme = 'dcterms.w3cdtf'
        elif scheme == 'iso8601':
          doc.log('NOTE', 'import', 'Deprecated date scheme', scheme)
        elif scheme != 'dcterms.w3cdtf':
          doc.log('WARN', 'import', 'Unknown date scheme', scheme)
        doc.setMetadata(DCTERMS.modified, Literal(content, datatype=DCTERMS.W3CDTF))

      # dc:publisher
      elif fld == 'dc.publisher':
        if not scheme:
          scheme = 'TS.FOAFOrganization'
          doc.log('NOTE', 'import', "No publisher scheme given, guessing %s" % scheme)
          scheme = scheme.lower()
        
        if scheme == 'ts.foaforganization':
          doc.setMetadata(DC.publisher, Literal(content, datatype=TS.FOAFOrganization))
        else:
          doc.log('WARN', 'import', 'Unknown publisher scheme, using TS.FOAFAgent', meta.attrib.get('scheme'))
          doc.setMetadata(DC.publisher, Literal(content, datatype=TS.FOAFAgent))

      # dc:creator
      elif fld in ('dc.creator', 'dc.creator.corporatename', 'dc.creator.personalname'):
        if fld != 'dc.creator':
          doc.log('NOTE', 'import', 'Deprecated field name', meta.attrib.get('name'))
        if not scheme:
          if fld == 'dc.creator.corporatename':
            scheme = 'TS.FOAFOrganization'
          elif fld == 'dc.creator.personalname':
            scheme = 'TS.FOAFPerson'
          elif content.find(',') != -1:
            scheme = 'TS.FOAFPerson'
          else:
            scheme = 'TS.FOAFOrganization'
          doc.log('NOTE', 'import', "No creator scheme given, guessing %s" % scheme)
          scheme = scheme.lower()
        
        if scheme == 'ts.foaforganization':
          doc.setMetadata(DC.creator, Literal(content, datatype=TS.FOAFOrganization))
        elif scheme == 'ts.foafperson':
          doc.setMetadata(DC.creator, Literal(content, datatype=TS.FOAFPerson))
        elif scheme == 'ts.foafcreator':
          doc.log('NOTE', 'import', 'Deprecated creator scheme, creating FOAF.Person instance', meta.attrib.get('scheme'))
          doc.setMetadata(DC.creator, Literal(content, datatype=TS.FOAFPerson))
        else:
          doc.log('WARN', 'import', 'Unknown creator scheme, creating FOAF.Agent instance', meta.attrib.get('scheme'))
          doc.setMetadata(DC.creator, Literal(content, datatype=TS.FOAFAgent))

      # ts:genre
      elif fld in ('ts.genre', 'dc.genre', 'dcterms.genre'):
        if fld != 'ts.genre':
          doc.log('NOTE', 'import', 'Deprecated field name', meta.attrib.get('name'))
        if not scheme:
          doc.log('NOTE', 'import', 'No genre scheme given, using TS.GENRE')
          scheme = 'ts.genre'
        schemeuri, matchtype = lookup.getSchemeURI(scheme)
        if matchtype == 'altname':
          doc.log('NOTE', 'import', 'Deprecated genre scheme', scheme)
        if not schemeuri:
          doc.log('WARN', 'import', 'Unknown genre scheme', scheme)
        doc.setMetadata(TS.genre, Literal(content, datatype=schemeuri))

      # dc:type
      elif fld == 'dc.type':
        if not scheme:
          doc.log('NOTE', 'import', 'No type scheme given, using DCTERMS.DCMIType')
          scheme = 'dcterms.dcmitype'
        schemeuri, matchtype = lookup.getSchemeURI(scheme)
        if matchtype == 'altname':
          doc.log('NOTE', 'import', 'Deprecated type scheme', scheme)
        if not schemeuri:
          doc.log('WARN', 'import', 'Unknown type scheme', scheme)
        if content.lower() == 'article': # workaround for www.tervesuomi.fi
          doc.log('NOTE', 'import', 'Deprecated DCMI type, using Text instead', content)
          content = 'Text'
        doc.setMetadata(DC.type, Literal(content, datatype=schemeuri))

      # dc:format
      elif fld == 'dc.format':
        if not scheme:
          doc.log('NOTE', 'import', 'No format scheme given, using DCTERMS.IMT')
        elif scheme == 'fileformat':
          doc.log('NOTE', 'import', 'Deprecated format scheme', scheme)
        elif scheme != 'dcterms.imt':
          doc.log('WARN', 'import', 'Unknown format scheme', scheme)
        doc.setMetadata(DC['format'], Literal(content, datatype=DCTERMS.IMT))

      # dc:subject
      elif fld == 'dc.subject':
        if not scheme:
          # store a simple literal subject, to be processed later
          doc.setMetadata(DC.subject, Literal(content, lang))
        else:
          schemeuri, matchtype = lookup.getSchemeURI(scheme)
          if matchtype == 'altname':
            doc.log('NOTE', 'import', 'Deprecated subject scheme', scheme)
          if not schemeuri:
            doc.log('WARN', 'import', 'Unknown subject scheme', scheme)
          if content.find("\n") != -1:
            doc.log('NOTE', 'import', 'Multiple subjects in single field')
            for c in content.split("\n"):
              c = c.strip()
              doc.setMetadata(DC.subject, Literal(c, datatype=schemeuri))
          else:
            doc.setMetadata(DC.subject, Literal(content, datatype=schemeuri))

      # ts:keyword
      elif fld == 'ts.keyword':
        doc.setMetadata(TS.keyword, Literal(content, lang))
      
      # dcterms:audience
      elif fld in ('dcterms.audience', 'dc.audience'):
        if fld != 'dcterms.audience':
          doc.log('NOTE', 'import', 'Deprecated field name', meta.attrib.get('name'))
        if not scheme:
          doc.log('NOTE', 'import', 'No audience scheme given, trying TS.SECTION')
          scheme = 'ts.section'
        schemeuri, matchtype = lookup.getSchemeURI(scheme)
        if matchtype == 'altname':
          doc.log('NOTE', 'import', 'Deprecated audience scheme', scheme)
        if not schemeuri:
          doc.log('WARN', 'import', 'Unknown audience scheme', scheme)
        if content.find("\n") != -1:
          doc.log('NOTE', 'import', 'Multiple audiences in single field')
          for c in content.split("\n"):
            c = c.strip()
            doc.setMetadata(DCTERMS.audience, Literal(c, datatype=schemeuri))
        else:
          doc.setMetadata(DCTERMS.audience, Literal(content, datatype=schemeuri))
        
      
      # dcterms:spatial
      
      # dcterms:temporal
      
      # dcterms:isPartOf
      
      # dc:rights
      elif fld == 'dc.rights':
        doc.setMetadata(DC.rights, Literal(content, lang))
#        if content.startswith('http://') or content.startswith('https://'):
#          doc.log('NOTE', 'import', 'Document references in DC.rights should use LINK not META')
#          doc.setMetadata(DC.rights, URIRef(content))
#        else:
      
      # dc:source
      
      # ts:isTranslationOf
      
      # dcterms:isFormatOf

      # ts:noindex
      elif fld == 'ts.noindex':
        doc.setMetadata(TS.noindex, Literal(content, datatype=XSD_NS.boolean))
      
      else:
        doc.log('NOTE', 'import', 'Unsupported field name', fld)

    # document is a ts:publication
    doc.setMetadata(RDF.type, TS.Publication)
    
    # mark the document as having completed processing
    doc.setCompleted('import')
    
    # schedule the next phase
    self._session.schedule(doc)
    
    return True
    
